# encoding: utf-8
__author__ = 'wwj'
import re
import sys
reload(sys)
sys.setdefaultencoding('utf8')
# sys.path.append("/Users/wwj/Downloads/shangtianjun/girl/girl/dbShell")
from dbshell import dbshell
from datetime import *
from zlib import crc32
import copy

class main(dbshell):
    orderItem = [
            ['seriesId', ''],
            ['directorId', ''],
            ['title', ''],
            ['name', ''],
            ['bwh', ''],
            ['designation', ''],
            ['cateId', ''],
            ['manufacturerId', ''],
            ['bigImg', ''],
            ['type', ''],
            ['series', ''],
            ['director', ''],
            ['cate', ''],
            ['version', ''],
            ['manufacturer', ''],
            ['thumb', ''],
            ['video', ''],
            ['img', ''],
    ]
    mongoTable = "movie"
    mysqlTable = "dmm_av"
    exesize = 500
    def __init__(self,spidername):
        super(main,self).__init__(spidername)
        # tmp = self.getdb("select id,name from dmm_play")
        # self.play = {x["name"]:x["id"] for x in tmp}
        if not self.redisconn.exists("series"):
            tmp = self.getdb("select id,name from dmm_series")
            self.redisconn.hmset("series",{x["name"]:x["id"] for x in tmp})
            self.redisconn.expire("series",timedelta(days=1))
        if not self.redisconn.exists("director"):
            tmp = self.getdb("select id,name from dmm_director")
            self.redisconn.hmset("director",{x["name"]:x["id"] for x in tmp})
            self.redisconn.expire("director",timedelta(days=1))
        if not self.redisconn.exists("manufacturer"):
            tmp = self.getdb("select id,name from dmm_manufacturer")
            self.redisconn.hmset("manufacturer",{x["name"]:x["id"] for x in tmp})
            self.redisconn.expire("manufacturer",timedelta(days=1))
        if not self.redisconn.exists("cate"):
            tmp = self.getdb("select id,name from dmm_cate")
            self.redisconn.hmset("cate",{x["name"]:x["id"] for x in tmp})
            self.redisconn.expire("cate",timedelta(days=1))
        if not self.redisconn.exists("dmmimg"):
            for x in self.client.dmm2.img.find({},{"source":1,"path":1},no_cursor_timeout=True).batch_size(1000):
                self.redisconn.hset("dmmimg",x["source"],x["path"])
                print x.get("source")
            self.redisconn.expire("dmmimg",timedelta(days=1))
        # self.geturltool("select distinct(url) from dmm_av")

    def clean(self,y):
        data = self.data
        data["seriesId"] = self.redisconn.hget("series",y.get("series"))
        data["directorId"] = self.redisconn.hget("director",y.get("director"))
        data["manufacturerId"] = self.redisconn.hget("manufacturer",y.get("manufacturer"))
        data["cateId"] = self.redisconn.hget("cate",y.get("tag"))
        data["name"] = y["name"]
        data["bwh"] = y["bwh"]
        data["designation"] = y.get("designation") or ""
        data["title"] = y["title"]
        if y.get("bigImg"):
            data["bigImg"] = self.redisconn.hget("dmmimg",y.get("bigImg"))
        else:
            data["bigImg"] = ""
        if y["url"].find('videoa') > 0:
            data['type'] = 1
        elif y["url"].find('videoc') > 0:
            data['type'] = 2
        elif y["url"].find('anime') > 0:
            data['type'] = 3
        elif y["url"].find('nikkatsu') > 0:
            data['type'] = 4
        data["url"] = y["url"].replace("http://www.dmm.co.jp","")
        data["urlHash"] = str(crc32(data["url"])).replace("-","")
        data["series"] = y.get("series")
        data["director"] = y.get("director")
        data["cate"] = y.get("tag")
        data["version"] = y.get("release")
        # data["apply"] = y.get("apply")
        data["manufacturer"] = y.get("manufacturer")
        if y.get("thumb"):
            result = self.client.dmm2.img.find_one({"type":"thumb","source":y.get("thumb")})
            if result:
                data["thumb"] = result.get("path")
            # data["thumb"] = self.redisconn.hget("dmmimg",y.get("thumb"))
        if not data.get("thumb"):
            raise Exception("no thumb" + y.get("thumb",""))
            # data["thumb"] = ""
        if y.get("video"):
            # data["video"] = self.redisconn.hget("dmmimg",y.get("video"))
            self.client.dmm2.img.find_one({"type":"video","source":yy})
        else:
            data["video"] = ""
        if y.get("imgList"):
            tmp = []
            for yy in y.get("imgList"):
                yy = yy.replace("-","jp-")
                result = self.client.dmm2.img.find_one({"type":"imgList","source":yy})
                if result:
                    img = result.get("path")
                # yy = yy.replace("-","jp-")
                # img = self.redisconn.hget("dmmimg",yy)
                    if img and img.find("mp4") < 0:
                        tmp.append(img)
            try:
                data["img"] = ",".join(tmp)
            except:
                # import ipdb;ipdb.set_trace()
                pass
        tmpdata = copy.deepcopy(data)
        self.datas.append(tmpdata)

    def buildsql(self,data):
        keys = self.data.keys()
        dd = []
        data_values = "(" + "%s," * (len(keys)) + ")"
        data_values = data_values.replace(',)', ')')
        keys = str(tuple(keys)).replace("'",'')
        values = [data]
        for value in values:
            dd = []
            sql = "insert ignore into %s %s values %s" % (self.mysqlTable,keys,data_values)
            for v in value:
                dd.append([MySQLdb.escape_string(str(y).strip().replace("%","")) for yy,y in v.iteritems()])
            try:
                pp(dd)
                self.conn.ping(True)
                self.cursor = self.conn.cursor()
                self.cursor.executemany(sql,dd)
                self.conn.commit()
                del self.datas[:self.exesize]
            except Exception as ex:
                import ipdb;ipdb.set_trace()
                self.conn.ping(True)
                print ex


if __name__ == '__main__':
    obj = main("dmm")
    obj()
